from __future__ import division
# Program inputs:
# (1) inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries_tail_90K_for_training"

# Program outputs:
# (1) NULL

# Output File Analysis:
# the file called freqOfTermsInQueries.txt contains 38871 unique query terms with their frequency in the query trace

from struct import *
from operator import itemgetter
import sys

print "Begin:"
# the input files should have 2
# one for queries that are human judged, and one for queries that are NOT just for effeciency task for the year 2006
# please add the special char filer for all our project

queryTermFreqDict = {}


inputFileNameList = []

# for training
inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries_tail_90K_for_training"
outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries_tail_90K_for_training_plain_text_only"

# for testing
# inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries_head_10K_for_testing"
# outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/100KQueries_head_10K_for_testing_plain_text_only"

outputFileHandler = open(outputFileName,"w")

# This file is used just for testing the filter to normalize the query terms, the only thing is to be consistent with what the polyIRToolkit is originally doing.
# inputFileName3 = "/home/obukai/workspace/polyIRToolkit/polyIRIndexer/difficultQueries.test"

inputFileNameList.append(inputFileName)
#inputFileNameList.append(inputFileName3)


for inputFileName in inputFileNameList:
    inputFileHandler = open(inputFileName,"r")
    
    for line in inputFileHandler.readlines():
        # print "line:",line.strip()
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        # print "queryTermList:",queryTermList
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        # print "data(old):",data
        # print "original data:",data
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
    
        # print "data(new):",data
        
        newQueryTermList = data.strip().split(" ")
        
        # print newQueryTermList
        
        for currentTerm in newQueryTermList:
            outputFileHandler.write(currentTerm + " ")
        
        outputFileHandler.write("\n")
        # Now connect them together
        
        '''
        for queryTerm in newQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in queryTermFreqDict:
                    queryTermFreqDict[queryTermLower] = 1
                else:
                    queryTermFreqDict[queryTermLower] += 1
        '''
 
'''
queryTermFreqList = []
queryTermProbabilityListNoSmoothing = []
 
for currentQueryTerm in queryTermFreqDict:
    queryTermFreqTuple = (currentQueryTerm, queryTermFreqDict[currentQueryTerm])
    queryTermFreqList.append(queryTermFreqTuple)

queryTermFreqList.sort(cmp=None, key=itemgetter(1), reverse=False)

numberOfQueryTerms = 0
numberOfUniqueQueryTerms = 0

for tuple in queryTermFreqList:
    (queryTerm,freq) = tuple
    numberOfQueryTerms += freq

numberOfUniqueQueryTerms = len(queryTermFreqList)

print "numberOfUniqueQueryTerms:",numberOfUniqueQueryTerms
print "numberOfQueryTerms:",numberOfQueryTerms

# Let's fill the queryTermDistributionDict
for tuple in queryTermFreqList:
    (queryTerm,freq) = tuple
    probability = freq / numberOfQueryTerms
    newTuple = (queryTerm,probability)
    queryTermProbabilityListNoSmoothing.append(newTuple)
    
queryTermProbabilityListNoSmoothing.sort(cmp=None, key=itemgetter(1), reverse=False)
for tuple in queryTermProbabilityListNoSmoothing:
    print tuple
'''

# outputFileHandler.write("\n")
outputFileHandler.close()               
inputFileHandler.close()
print "End..."

